import requests
from bs4 import BeautifulSoup
import math
from pymongo import MongoClient
import urllib.parse
import crawler
import time
baseurl='http://tieba.baidu.com/f/like/furank?kw={tiebaname}&ie=utf-8&pn={pagenum}'
#baseurl='http://tieba.baidu.com'


client=MongoClient('localhost',27017)
db=client.cache


if __name__ == '__main__':    
    # threaded_crawler(10)
    #proxy={ "http": "http://10.10.1.10:3128"}
    #proxy=crawler.getproxy()
    proxy=None
    tiebaname="新宋吧"
    url=baseurl.format(tiebaname=urllib.parse.quote(tiebaname),pagenum=1)
    response=requests.get(url,proxies=proxy)
    soup = BeautifulSoup(response.text,"html.parser")
    
    qty_ele=soup.find(attrs={'class':'drl_info_txt_gray'})
    usernum=int(qty_ele.text)
    for i in range(533,math.ceil(usernum/20)+1):
        time.sleep(1)
        print(i)
        url=baseurl.format(tiebaname=urllib.parse.quote(tiebaname),pagenum=i)
        
        html=requests.get(url,proxies=proxy)
        bs=BeautifulSoup(html.text,'html.parser')
        namelist = bs.find_all(attrs={'class':'drl_item_name'})
        levellist = bs.find_all(attrs={'class':'drl_item_title'})
        jiangyanlist=bs.find_all(attrs={'class':'drl_item_exp'})
        x=-1
        for l in namelist:
            print(l.text)
            x=x+1
            db.tieba_user.insert({
                'tieba':tiebaname,
                'name':l.text,
                'level':int(levellist[x].next.attrs['class'][0][5:]),
                'jingyan':int(jiangyanlist[x].text),
                'status':0
            })
            # if db.tiebauser.find({'_id':l.text}):
            #     print('已存在')
            # else:
            if l.text:
                db.tiabuser1.insert({'_id':l.text,'status':0})
                 if db.tiebauser.find({'_id':l.text}):
                print('已存在')
            else:
                db.tiebauser.insert({'_id':l.text,'status':0})
    #while flg:
       
        # if db.tiebauser.find({'_id':i['user']}):
        #     print('已存在')
        # else:
        #     db.tiabuser.insert({'_id':i['user']})